#include "phraser.h"

using namespace std;
using namespace mongo;
using namespace re2;
using namespace zxlib;

phraser::phraser(cmd_line& cmd_line)
{
     m_cmd_line = cmd_line;
     char conn_str[128];
     zxlib::load_yaml(cmd_line.config_file.c_str(),m_config);
     m_filter.load_filter(m_config.filter_file.c_str());
     sprintf(conn_str,"%s:%d",m_config.host.c_str(),m_config.port);
     m_connection_string=conn_str;
     m_connection.connect(m_connection_string);
}
void phraser::by_length()
{
     int count = 0;
     auto_ptr<DBClientCursor> cursor = m_connection.query(m_config.collection, m_config.query_obj,m_cmd_line.limit);

     cout << "counting \t" << m_config.collection << "." << m_config.field << "@" << m_connection_string << endl ;
     while( cursor->more() )
     {
          const char *atext = (cursor->next()).getStringField(m_config.field.c_str());
          if( Utf8Iterator(atext) > 3 ||  filter_word(atext) ){
               continue;
          }
          m_word_map[string(atext)]++;
          if (++count % 100000 == 0)
          {
               cout << count << " words:" << m_word_map.size() << endl;
          }
     }
     cout << "count over:" << m_word_map.size() << " words" << endl;
     save_to_file(m_word_map,m_cmd_line.output_file.c_str());
}
void print_terms( term_array &ret ){
     for (size_t i = 0; i < ret.size(); i++){
          cout << " " << ret[i].term_text << "[" << ret[i].term_type << "]";
     }
     cout << endl;
}
bool phraser::stats( term_array& terms , const char* org_text )
{
     string_vector strs;
     size_t tmp=0;
     for( size_t n = 0 ; n < terms.size() ; n ++ ){
          strs.push_back(terms[n].term_text);
          if (terms[n].term_text.size() == 3){
               tmp+=1;
          }
     }
     vector<string_vector> found;
     size_t deltas[] ={2,3};
     size_t count = zxlib::find_isolate_chars(strs,found);
     if( count > 0 ){
          for( size_t n =0 ; n < found.size() ; n++){
               if( found[n].size() < 2 ) continue;
               string_vector groups;
               for( size_t i=0 ; i< sizeof(deltas)/sizeof(size_t);i++ ){
                    if( found[n].size() < deltas[i] ) continue;
                    count = group_chars( found[n],deltas[i],groups);
                    if( count > 0 ){
                         for( size_t k = 0 ; k< groups.size() ; k++ ){
                              m_word_map[groups[k]]++;
                         }
                    }
                    groups.clear();
               }
          }
     }
     return tmp > 0;
}
void phraser::by_stats()
{
     int count = 0;
     auto_ptr<DBClientCursor> cursor = m_connection.query(m_config.collection, m_config.query_obj,m_cmd_line.limit);

     term_array terms;
     cout << "counting \t" << m_config.collection << "." << m_config.field << "@" << m_connection_string << endl ;
     while( cursor->more())
     {
          BSONObj doc = cursor->next();
          const char *text = doc.getStringField(m_config.field.c_str());
          BSONElement e ;
          doc.getObjectID(e);
          zstmt stmt = ztexting::instance()->parse_stmt(text);
          terms =stmt.get_terms();
          if(stats(terms,text)){
               //cout << e.toString() <<":"<< text << "==>" ;
               //print_terms(terms);
          }
          if (++count % 100000 == 0)
          {
               cout << count << " words:" << m_word_map.size() << endl;
          }
     }
     cout << "count over:" << m_word_map.size() << " words" << endl;
     save_to_file(m_word_map,m_cmd_line.output_file.c_str());
}
bool compareWordCount(const Item *a, const Item *b)
{
     //sort by asc
     return a->second < b->second;
}
void phraser::save_to_file( Dict& words , const char* filename)
{
     ofstream out;
     out.open(filename);
     vector<Item*> vec;
     vec.reserve(words.size());
     for(Dict::iterator i = words.begin(); i != words.end(); i++)
     {
          vec.push_back(&(*i));
     }
     sort(vec.begin(), vec.end(), compareWordCount);
     for(vector<Item*>::iterator i = vec.begin(); i != vec.end(); i++)
     {
          cout << (*i)->first << " : " << (*i)->second << endl;
          out << (*i)->first << " : " << (*i)->second << endl;
     }
}
bool phraser::filter_word( const char* word )
{
     if(RE2::PartialMatch(word,"[a-zA-Z0-9]+")){
          return true;
     }
     if(RE2::PartialMatch(word, m_filter.get_pattern())) {
          return true;
     }
     return false;
}
